package com.yaoyong.yy_demo.module.webmagic;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.ResultItems;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.monitor.SpiderMonitor;
import us.codecraft.webmagic.pipeline.Pipeline;
import us.codecraft.webmagic.processor.PageProcessor;

import javax.management.JMException;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;

/**
 * 爬取4k壁纸网的图片
 */
public class PicturePageProcessor implements PageProcessor {

    /**
     * 设置参数:抓取网站的相关配置，包括编码、抓取间隔、重试次数等
     */
    private Site site = Site.me().setRetryTimes(3).setSleepTime(1000).setTimeOut(3000);

    /**
     * process是定制爬虫逻辑的核心接口，在这里编写抽取逻辑
     */
    @Override
    public void process(Page page) {
        // 一页抓20张图片
        for (int i = 1; i < 21; i++) {
            // 定义如何抽取页面信息，并保存下来
            String singlePage = page.getHtml().xpath("//div[@class='slist']/ul[@class='clearfix']/li[" + i + "]/a/@href").toString();
            page.addTargetRequest(singlePage);
        }
        String imageUrl = page.getHtml().xpath("//div[@class='photo-pic']/a/img/@src").toString();
        String title = page.getHtml().xpath("//div[@class='photo-pic']/a/img/@title").toString();
        if (null !=imageUrl) {
            page.putField("imageUrl", "http://pic.netbian.com" + imageUrl);
            page.putField("title", title + ".jpg");
        }
    }


    @Override
    public Site getSite() {
        return site;
    }

    /**
     * 主方法启动爬虫
     */
    public static void main(String[] args) throws JMException {
        SpiderMonitor instance = SpiderMonitor.instance();
        //Spider是爬虫的入口类。Pipeline是结果输出和持久化的接口
        // 这里只爬取第一页的壁纸，如果要爬取其他页数修改for循环参数即可
        for (int i = 1; i <= 72; i++) {
            // 启动爬虫
            Spider spider = Spider.create(new PicturePageProcessor())
                    // 添加初始化的URL
                    .addUrl(i == 1 ? "http://pic.netbian.com/4kbeijing/index.html" : "http://pic.netbian.com/4kbeijing/index_" + i + ".html")
                    .addPipeline(new MyPipeline())
                    // 使用单线程
                    .thread(2);
            instance.register(spider);
                    // 运行
            spider.start();
        }

    }
}

class MyPipeline implements Pipeline {

    @Override
    public void process(ResultItems resultItems, Task task) {
        try {
            DownloadImage.downLoadFromUrl(resultItems.get("imageUrl"), resultItems.get("title"), "D:\\download\\crawler\\4kbeijing");
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}

class DownloadImage {

    public static void downLoadFromUrl(String urlStr, String fileName, String savePath) throws IOException {
        URL url = new URL(urlStr);
        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
        // 设置超时间为3秒
        conn.setConnectTimeout(3 * 1000);

        // 防止屏蔽程序抓取而返回403错误
        conn.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt)");

        // 得到输入流
        InputStream inputStream = conn.getInputStream();

        // 获取自己数组
        byte[] getData = readInputStream(inputStream);

        // 文件保存位置
        File saveDir = new File(savePath);
        if (!saveDir.exists()) {
            saveDir.mkdir();
        }

        File file = new File(saveDir + File.separator + fileName);
        FileOutputStream fos = new FileOutputStream(file);
        fos.write(getData);
        if (fos != null) {
            fos.close();
        }

        if (inputStream != null) {
            inputStream.close();
        }

        System.out.println("info:" + url + " download success");

    }

    private static byte[] readInputStream(InputStream inputStream) throws IOException {
        byte[] buffer = new byte[1024 * 8];
        int len = 0;
        ByteArrayOutputStream bos = new ByteArrayOutputStream();
        while ((len = inputStream.read(buffer)) != -1) {
            bos.write(buffer, 0, len);
        }
        bos.close();
        return bos.toByteArray();
    }
}

